/**
 * Copyright (C), 2015-2021, 开度
 * FileName: SpiderUtils
 * Author:   ASUS
 * Date:     2021/5/7 17:03
 * Description: 爬虫工具
 * History:
 * <author>          <time>          <version>          <desc>
 * Hezeyu           2021/5/7           1.0              爬虫工具
 */
package com.example.spring.security.utils;

import com.example.spring.security.utils.http.HttpUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.URL;
import java.util.Date;

/**
 * 〈爬虫工具〉
 *
 * @author ASUS
 * @create 2021/5/7
 * @since 1.0.0
 */
@Slf4j
@Data
public class SpiderUtils {

    /**
     * 需要爬的网站地址
     */
    public static String path = "https://tieba.baidu.com/p/5104408604";

    public static String noPath = "lol.qq.com";

    public static int num = -1, sum = 0;

    /**
     * 需要存放的地址
     */
    public static final String WORKSPACE = "src/main/resources/spieder";

    /**
     * 定义四个文件类（链接存储，图片储存，文件存储，错误链接存储）
     */
    public static File aLinkFile, imgLinkFile, docLinkFile, errorLinkFile;

    /**
     * 开始爬取
     *
     * @param path
     * @return:void
     * @since: 1.0.0
     * @Author:hezeyu
     * @Date: 2021/5/7 17:39
     */
    public static void getAllLinks(String path) {
        Document doc = null;
        try {
            doc = Jsoup.parse(HttpUtils.get(path,"UTF-8"));
        } catch (Exception e) {
            write(errorLinkFile, path + "\n");
            num++;
            if (sum > num) {
                getAllLinks(getFileLine(aLinkFile, num));
            }
            return;
        }
        Elements aLinks = doc.select("a[href]");
        Elements imgLins = doc.select("img[src]");

        log.info("开始链接：" + path);
        aLinks.forEach(item -> {
            String url = item.attr("href");
            //判断链接是否包含这两个头
            if (!url.contains("http://") && !url.contains("https://") && !url.contains("//")) {
                //不是则加上	例：<a href="xitongshow.php?cid=67&id=113" />
                //则需要加上前缀	http://www.yada.com.cn/xitongshow.php?cid=67&id=113
                //否则404
                url = SpiderUtils.path + url;
            }
            if (!readTxtFile(aLinkFile).contains(url) && !url.contains("javascript")) {
                if (url.contains(SpiderUtils.path)) {
                    //判断该a标签的内容是文件还是子链接
                    if (url.contains(".doc") || url.contains(".exl")
                            || url.contains(".exe") || url.contains(".apk")
                            || url.contains(".mp3") || url.contains(".mp4")) {
                        //写入文件中，文件名+文件链接
                        write(docLinkFile, item.text() + "\r\n\t" + url + "\r\n");
                    } else {
                        //将链接写入文件
                        write(aLinkFile,  item.text() + "：\t" + url + "\r\n");
                        sum++;    //链接总数+1
                    }
                    log.info("\t" + item.text() + "：\t" + url);
                }
            }
        });
        imgLins.forEach(img -> {
            String imgSrc = img.attr("src");
            if (!imgSrc.contains("http://") && !imgSrc.contains("https://") && !imgSrc.contains("//")) {
                imgSrc = SpiderUtils.path + imgSrc;
            }else if(!imgSrc.contains("http") || !imgSrc.contains("https")){
                imgSrc="http:"+imgSrc;
            }
            if (!readTxtFile(imgLinkFile).contains(imgSrc)) {
                //将图片链接写进文件中
                Download(imgSrc);
                write(imgLinkFile, imgSrc + "\r\n");
            }
        });
        num++;
        if (sum > num) {
            getAllLinks(getFileLine(aLinkFile, num));
        }

    }

    public static void main(String[] args) {
        //判断文件夹是否存在，不存在则创建
        File directory = new File(WORKSPACE);
        if (!directory.exists() && !directory.isDirectory()) {
            directory.mkdir();
        }
        aLinkFile = new File(WORKSPACE + "/ALinks.txt");
        imgLinkFile = new File(WORKSPACE + "/ImgLinks.txt");
        docLinkFile = new File(WORKSPACE + "/DocLinks.txt");
        errorLinkFile = new File(WORKSPACE + "/ErrorLinks.txt");
        //用数组存储四个文件对象，方便进行相同操作
        File[] files = new File[]{aLinkFile, imgLinkFile, docLinkFile, errorLinkFile};
        try {
            for (File file : files) {
                if (file.exists())    //如果文件存在
                    file.delete();    //则先删除
                file.createNewFile();    //再创建
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        long startTime = System.currentTimeMillis();    //获取开始时间
        getAllLinks(path);
        int imgCount = getFileCount(imgLinkFile);
        int docCount = getFileCount(docLinkFile);
        write(aLinkFile, "链接总数：" + getFileCount(aLinkFile) + "条");
        write(imgLinkFile, "图片总数：" + imgCount + "张");
        write(docLinkFile, "文件总数：" + docCount + "份");
        write(errorLinkFile, "问题链接总数：" + getFileCount(errorLinkFile) + "条");

        log.info(""
                + "——————————————————爬取结束——————————————————"
                + "\n目标网址：" + path
                + "\n链接总数：" + sum + "条"
                + "\n图片总数：" + imgCount + "张"
                + "\n文件总数：" + docCount + "份");

        long endTime = System.currentTimeMillis();    //获取结束时间
        log.info("\n程序运行时间：" + (endTime - startTime) + "ms");
    }

    /**
     * 下载图片或者文件
     *
     * @param url 图片路径
     * @return:void
     * @since: 1.0.0
     * @Author:hezeyu
     * @Date: 2021/5/8 9:46
     */
    private static void Download(String url) {
        try {
            //开始时间
            Date begindate2 = new Date();
            String imageName = url.substring(url.lastIndexOf("/") + 1, url.length());
            URL uri = new URL(url);
            InputStream in = uri.openStream();
            File file=new File("src/main/resources/img/"+ imageName);
            if(file.exists()){
                file.delete();
                file.createNewFile();;
            }
            FileOutputStream fo = new FileOutputStream(file);//文件输出流
            byte[] buf = new byte[1024];
            int length = 0;
            System.out.println("开始下载:" + url);
            while ((length = in.read(buf, 0, buf.length)) != -1) {
                fo.write(buf, 0, length);
            }
            //关闭流
            in.close();
            fo.close();
            System.out.println(imageName + "下载完成");
            //结束时间
            Date overdate2 = new Date();
            double time = overdate2.getTime() - begindate2.getTime();
            System.out.println("耗时：" + time / 1000 + "s");
        } catch (Exception e) {
            log.error(e.getMessage());
        }
    }

    /**
     * 获取文件总行数（有多少链接）
     *
     * @param file 文件路径
     * @return:int 文件数量
     * @since: 1.0.0
     * @Author:hezeyu
     * @Date: 2021/5/7 17:39
     */
    public static int getFileCount(File file) {
        int count = 0;
        try {
            BufferedReader reader = new BufferedReader(new FileReader(file));
            while (reader.readLine() != null) {    //遍历文件行
                count++;
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return count;
    }

    /**
     * 读取文件内容
     *
     * @param file 文件路径
     * @return:java.lang.String 文件内容
     * @since: 1.0.0
     * @Author:hezeyu
     * @Date: 2021/5/7 17:39
     */
    public static String readTxtFile(File file) {
        String result = "";        //读取結果
        String thisLine = "";    //每次读取的行
        try {
            BufferedReader reader = new BufferedReader(new FileReader(file));
            try {
                while ((thisLine = reader.readLine()) != null) {
                    result += thisLine + "\n";
                }
                reader.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
        return result;
    }

    /**
     * 获取文件指定行数的数据，用于爬虫获取当前要爬的链接
     *
     * @param file 目标文件
     * @param num  指定的行数
     */
    public static String getFileLine(File file, int num) {
        String thisLine = "";
        int thisNum = 0;
        try {
            BufferedReader reader = new BufferedReader(new FileReader(file));
            while ((thisLine = reader.readLine()) != null) {
                if (num == thisNum) {
                    return thisLine;
                }
                thisNum++;
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return "";
    }

    /**
     * 写入文件
     *
     * @param file   文件路径
     * @param urlStr 需要写入的字符
     * @return:
     * @since: 1.0.0
     * @Author:hezeyu
     * @Date: 2021/5/7 17:09
     */
    public static void write(File file, String urlStr) {
        try {
            BufferedWriter writer = new BufferedWriter(new FileWriter(file, true));
            writer.write(urlStr);
            writer.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
